package org.nlp.algo.segment;

import java.io.IOException;
import java.util.Iterator;

import org.nlp.algo.classifier.LuceneTokenizerFactory;

import com.aliasi.tokenizer.Tokenizer;

/**
 * @author longkeyy
 * 分词演示
 *
 */
public class tokenizerFactoryDemo {
	public static void main(String[] args) throws IOException {

		char[] ch = "中华人民共和国".toCharArray();
		int length = ch.length;
		int start = 0;

		LuceneTokenizerFactory tokenizerFactory = LuceneTokenizerFactory.INSTANCE;
		Tokenizer tokenizer = tokenizerFactory.tokenizer(ch, start, length);

		// IndoEuropeanTokenizerFactory indoEuropeanTokenizerFactory = new
		// IndoEuropeanTokenizerFactory();
		// IndoEuropeanTokenizerFactory instance =
		// indoEuropeanTokenizerFactory.INSTANCE;
		// Tokenizer tokenizer = instance.tokenizer(ch, start, length);

		Iterator<String> iterator = tokenizer.iterator();
		while (iterator.hasNext()) {
			System.out.println(iterator.next());
		}
	}
}